#!/usr/bin/env python

"""proclog - process log files

The config file is a standard INI-format config file. Each section
represents a log file processor.

There are a number of options that control what is processed. The most
important of these is "match". This is a Python regular expression (see
the Python re module documentation) that each line being processed will be
matched against. If match is missing, an empty string will be used, which
matches every line. If the regular expression matches, the line will be
processed by this processor. If the line doesn't match, then it won't
be processed.

Normally, the entire line matched is processed by a processor. However,
two options can change that behavior if they are present: "format" is a
python %-format string that will be used with a dictionary of group names
from the "match" expression. "group" is the name of a match group(+) in the
regular expression, and "group = id" is a shorthand for "format = %(id)s".
Strings of the from '%(name)s' in the format string will be replaced by
the value of the match group 'name'. A '%%' will be replaced by a single
'%'. The result of formatting the string with the match dictionary is what
will be processed by this processor.

The processors are related to each other in a tree structure, specified
by the "process" option in each section. This option lists the section
names for the children of this node in the tree.

The roots of the trees are selected by the command line. If arguments
are passed to proclog, they will each have "proclog " prepended, and the
section of that name will be used as a root for a processing tree(*). If no
arguments are specified, then the section "proclog" will be used as the root
of the sole tree.

A node is a leaf node if the "process" option has only one entry, and that
entry is for a builtin processor (meaning it starts with "builtin "). How a
builtin processor processes a line depends on the type. This information is
available via the '--doc' flag to proclog.

The behavior of non-leaf nodes depend on the presence or absence of the
"file" or "command" options (a "source"), and the position of the node
in the tree. Starting from the root of a tree, nodes that don't have a
source will just run all their children. When a processor with a
source option is run, it will read log lines from the source and
process them, rather than running it's children. A processor that is not
a leaf node processes a line by passing it - or the formated version - to
each of it's children in turn, until one of them processes the line.

An optional "present" option is a comma-seperated list of section names from
the "process" list. The results will be printed in the order of the "present"
option. A section name that is on the "process" list but not the "present" list
will be used in processing, but it's results will not be printed. If an option
appears on the "present" list but not the "process" list, you'll get a run-time
error.

The "title" option will be used as the label for the output from this processor.

The "display" option controls how many lines of data will be output. If it's 0,
the processor will only display it's count. If it's None, all the data  lines
will be output. This is useful with the uniq builtin, to only see the N most
frequently occuring line.

Normally, if a processor processes 0 lines, it won't display anything. The
"expect" option can be used to change that. Set it to an integer, and the
processor won't display anything if that is what it's count is. This can be
used to filter out "expected" events that you don't want to ignore except for
those you expect. If you get just the ones you expect, nothing is displayed. If
you get any extra, or don't get the ones you expect, you get a display.

+) A match group has the form '(?P<name>...)', and the string matched by
the regular expression represented by '...' will be the value for that match
group. See the Python re module docs for a more complete discussion. See
the provided proclog.conf for some examples.

*) Actually, those sections will be made children of a processing node
that will simply run them all."""


import sys
import re
from os import popen
from ConfigParser import SafeConfigParser as ConfigParser
from optparse import OptionParser

from ConfigDict import ConfigDict


def main():
    """Figure out which sections we're going to process."""

    usage = '%prog --doc [processor ...] | [--config file] [name ... ]'
    opts = OptionParser(usage=usage)
    opts.add_option('-c', '--config', action='store',
                    help='Name of config file to use',
                    default='/usr/local/etc/proclog.conf')
    opts.add_option('-d', '--doc', help="Ask for documentation.",
                    action="store_true")

    flags, args = opts.parse_args()
    if flags.doc:
        help(args) 
        if not args:
            opts.print_help()
        return 0

    if args:
        sections = ["proclog %s" % arg for arg in args]
    else:
        sections = ["proclog"]

    config = ConfigParser()
    config.read(flags.config)
    proc = CommandLine(config, sections)
    proc.run()
    print
    print proc
        
    return 0


def help(args):
    """Print the help message."""

    if not args:
        print __doc__
        print
        print "Processors available:", ', '.join(builtins)
        print
    else:
        for arg in args:
            print builtins[arg].__doc__
            print


class Process(object):
    """A base class for processes."""

    def __init__(self, section):
        self.title = section.get('title')
        self.match = re.compile(section.get('match', '').strip()).search

        self.top = section.get('display')
        if self.top:
            self.top = int(self.top)


        expect = section.get('expect')
        if expect:
            self.expect = int(expect)
        else:
            self.expect = 0

        if section.has_key('group'):
            self.format = '%%(%s)s' % section['group'].strip()
        else:
            self.format = section.get('format', raw=True)

        self._source = None
        if section.has_key('file'):
            self._source = open, section['file']
        elif section.has_key('command'):
            self._source = popen, section['command']

    def run(self):
        datain = self._source[0](self._source[1])
        for line in datain:
            self.proc(line)

    def proc(self, line):
        m = self.match(line)
        if not m:
            return False
        
        if self.format:
            add = self.format % m.groupdict() + '\n'
        else:
            add = line

        return self._add(add)

    def _add(self, line):
        for proc in self._subprocs:
            if proc.proc(line):
                return True
        return False

    def __str__(self):
        # Present?
        if self.count == self.expect:
            return ''

        out = self.data[:self.top]
        if self.title:
            out.insert(0, '%s: %d\n' % (self.title, self.count))

        return ''.join(out)


class Counter(Process):
    """counter - count the the processed lines."""

    def __init__(self, section):
        super(Counter, self).__init__(section)
        self.count = 0
        self.data = []

    def _add(self, line):
        self.count += 1
        return True


class Lister(Counter):
    """lister - list all the processed strings in order."""

    def _add(self, key):
        self.count += 1
        self.data.append(key)
        return True


class Uniq(Process):
    """uniq - counts the number of occurences of each string processed, sorted by frequncy."""

    def __init__(self, section):
        super(Uniq, self).__init__(section)
        self.count = 0
        self._dict = {}

    def _add(self, line):
        self.count += 1
        self._dict[line] = self._dict.get(line, 0) + 1
        return True
            
    def _data(self):
        data = [(count, line) for (line, count) in self._dict.items()]
        data.sort(reverse=True)
        return [('%6d: %s' % item) for item in data]
    data = property(_data)
        

class LineControl(Process):
    """A Process that passes lines on to subprocesses."""

    def __init__(self, section, config):
        super(LineControl, self).__init__(section)

        sections = from_csl(section['process'])
        if section.has_key('present'):
            self._present = from_csl(section['present'])
        else:
            self._present = sections
            
        self._make_proctables(config, sections)

    def _make_proctables(self, config, sections):
        self._proctable = {}
        self._subprocs = []
        for sect in sections:
            proc = process_factory(config, sect)
            self._subprocs.append(proc)
            self._proctable[sect] = proc
        
    def _count(self):
        return sum([subproc.count for subproc in self._subprocs])
    count = property(_count)

    def _data(self):
        return sum([self._proctable[subproc].data for subproc in self._present],
                   [])
    data = property(_data)

    def __str__(self):
        if self.count == self.expect:
            return ''

        out = []
        if self.title:
            out.append('%s: %d\n' % (self.title, self.count))

        for sect in self._present:
            res = str(self._proctable[sect])
            if res:
                out.append(res)

        return '\n'.join(out)

class PureControl(LineControl):
    """A Control process that just runs subprocesses, without processing lines."""

    def run(self):
        for proc in self._subprocs:
            proc.run()


class CommandLine(PureControl):
    """A Process to control an externally-provided list of Processes."""

    def __init__(self, config, sections):
        self.title = "Total lines processed"
        self.expect = 0
        self.top = None
        self._present = sections
        self._make_proctables(config, sections)


def from_csl(line):
    return [x.strip() for x in line.split(',')]


builtins = dict(lister=Lister, counter=Counter, uniq=Uniq)
def process_factory(config, name):
    data = ConfigDict(config, name)
    procs = from_csl(data['process'])
    names = procs[0].split()
    if len(procs) == 1 and names[0] == 'builtin':
        return builtins[names[1]](data)
    if data.has_key('file') or data.has_key('command'):
        return LineControl(data, config)
    return PureControl(data, config)


if __name__ == '__main__':
    sys.exit(main())


